package reptile;

import com.google.gson.Gson;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.*;

/**
 * 标准地址库 数据来源于 国家统计局
 */
public class StandardAddress {

    private static final String BaseUrl = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/";
    private static final Gson gson = new Gson();

    /**
     * 开始爬数据 只处理了三级 一共有五级，需要自己处理
     */
    public static void main(String[] args) {
        //省列表(一级)
        List<AddressLibrary> privinceUrl;
        try {
            privinceUrl = parseUrl(BaseUrl, ClassEnum.PROVINCE);
        } catch (Exception e) {
            throw new RuntimeException("获取省列表[" + BaseUrl + "]失败！" + e.getMessage());
        }
        System.out.println("一级地址数量:" + privinceUrl.size());

        //市列表(二级)
        List<AddressLibrary> cityUrl = new ArrayList<>();
        boolean privinceStatus = true;
        int cityCount = 1;
        while (privinceStatus) {
            System.out.println("第[" + cityCount + "]次读取二级列表。");
            privinceUrl.stream().filter(x -> !x.isRead()).forEach(y -> {
                try {
                    List<AddressLibrary> sub = parseUrl(BaseUrl + y.getUrl(), ClassEnum.CITY);
                    String parentNode = sub.get(0).getId().substring(0, 2) + "0000000000";//父节点
                    sub.forEach(x -> {
                        x.setPid(parentNode);
                        cityUrl.add(x);
                    });
                    y.setRead(true);
                    y.setId(parentNode);
                    y.setPid("");
                } catch (Exception e) {
                    System.out.println("二级地址ERROR:" + BaseUrl + y.getUrl());
                }
            });
            privinceStatus = privinceUrl.stream().anyMatch(x -> !x.isRead());
            cityCount++;
        }
        System.out.println("二级地址数量:" + cityUrl.size());

        //区列表(三级)
        List<AddressLibrary> areaUrl = new ArrayList<>();
        boolean cityStatus = true;
        int areaCount = 1;
        while (cityStatus) {
            System.out.println("第[" + areaCount + "]次读取三级列表。");
            cityUrl.stream().filter(x -> !x.isRead()).forEach(y -> {
                try {
                    List<AddressLibrary> sub = parseUrl(BaseUrl + y.getUrl(), ClassEnum.AREA);
                    sub.forEach(x -> {
                        x.setPid(y.getId());
                        areaUrl.add(x);
                    });
                    y.setRead(true);
                } catch (Exception e) {
                    System.out.println("三级地址ERROR:" + BaseUrl + y.getUrl());
                }
            });
            cityStatus = cityUrl.stream().anyMatch(x -> !x.isRead());
            areaCount++;
        }
        System.out.println("三级地址数量:" + areaUrl.size());


        System.out.println("===============JSON测试数据");
        System.out.println("var provinceArr =" + gson.toJson(privinceUrl) + ";");
        System.out.println("var cityArr = " + gson.toJson(cityUrl) + ";");
        System.out.println("var areaArr =" + gson.toJson(areaUrl) + ";");

        System.out.println("===============合并数据");
        privinceUrl.addAll(cityUrl);
        privinceUrl.addAll(areaUrl);

        System.out.println("===============生成JS文件");
        String fileBasePath = "d:/address.";
        File jsFile = new File(fileBasePath + "js");
        if (jsFile.exists())
            jsFile.delete();
        try {
            jsFile.createNewFile();
            BufferedWriter writer = new BufferedWriter(new FileWriter(jsFile, true));
            writer.write(gson.toJson(privinceUrl));
            writer.close();
            System.out.println("生成路径:" + fileBasePath + "js");
        } catch (IOException e) {
            e.printStackTrace();
        }

        System.out.println("===============生成xlsx文件");
        LinkedHashMap<String, String> headerMap = new LinkedHashMap<>();
        headerMap.put("id", "编号");
        headerMap.put("pid", "上级编号");
        headerMap.put("name", "名称");
        headerMap.put("shortName", "简称");
        headerMap.put("addLevel", "级别");
        headerMap.put("delFlag", "状态");

        List<Map<String, Object>> dataMap = new ArrayList<>();
        privinceUrl.forEach(x -> {
            String strJson = gson.toJson(x);
            Map map = gson.fromJson(strJson, Map.class);
            dataMap.add(map);
        });
        ExportExcelUtil.ExportExcel(headerMap, dataMap, fileBasePath + "xlsx");
    }


    /**
     * 单个地址测试
     */
    @Test
    public void SingleAddress() {
        try {
            System.out.println(parseUrl("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/44/4419.html", ClassEnum.AREA));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    //解析URL内容
    private static List<AddressLibrary> parseUrl(String url, ClassEnum className) throws IOException {
        //System.out.println("解析地址:" + url);
        List<AddressLibrary> list = new ArrayList<>();

        //===1.创建连接
        //方式一 这种方式有可能会出现乱码(有部分内容)
        //Document doc = Jsoup.connect(url).timeout(3000).get();
        //方式二 解决乱码
        HttpURLConnection connection = (HttpURLConnection)new URL(url).openConnection();
        connection.setRequestMethod("GET");//默认就是Get，可以采用post，大小写都行，因为源码里都toUpperCase了。
        connection.setUseCaches(Boolean.FALSE);//是否允许缓存，默认true。
        //是否开启输入输出，如果是post使用true。默认是false
        //connection.setDoInput(Boolean.TRUE);
        //connection.setDoOutput(Boolean.TRUE);
        connection.addRequestProperty("Connection", "close");//设置请求头信息
        connection.setConnectTimeout(3000);//设置连接主机超时（单位：毫秒）
        connection.setReadTimeout(8000);//设置从主机读取数据超时（单位：毫秒）
        //connection.addRequestProperty("Cookie","你的Cookies" );//设置Cookie
        Document doc = Jsoup.parse(connection.getInputStream(), "GBK", url);//开始请求

        //===2.选择指定元素
        //Elements elements = doc.select(className.getClassName());
        Elements elements = splitClassName(doc, className.getClassName());
        for (Element element : elements) {
            Elements a = element.select("a");
            switch (className) {
                case PROVINCE:
                    a.forEach(x -> list.add(
                            new AddressLibrary()
                                    .setName(x.text())
                                    .setShortName(x.text())
                                    .setAddLevel(1)
                                    .setDelFlag(0)
                                    .setUrl(x.attr("href"))
                            )
                    );
                    break;
                case CITY:
                    list.add(
                            new AddressLibrary()
                                    .setId(a.get(0).text())
                                    .setName(a.get(1).text())
                                    .setShortName(a.get(1).text())
                                    .setAddLevel(2)
                                    .setDelFlag(0)
                                    .setUrl(a.get(1).attr("href"))

                    );
                    break;
                case AREA:
                    //System.out.println(a.size());
                    if (a.size() == 2)
                        list.add(
                                new AddressLibrary()
                                        .setId(a.get(0).text())
                                        .setName(a.get(1).text())
                                        .setShortName(a.get(1).text())
                                        .setAddLevel(3)
                                        .setDelFlag(0)
                                        .setUrl(a.get(1).attr("href"))
                        );
                    break;
                default:
                    throw new RuntimeException("未处理的节点！");
            }
        }
        return list;
    }

    //处理多个类名
    private static Elements splitClassName(Document doc, String classNameStr) {
        List<String> list = new ArrayList<>();
        if (!classNameStr.contains("|")) {
            list.add(classNameStr);
        } else {
            list.addAll(Arrays.asList(classNameStr.split("\\|")));
        }
        for (String className : list) {
            Elements elements = doc.select(className);
            if (elements.size() > 0)
                return elements;
        }
        return null;
    }
}
